import scrapy
from ..items import FormDataItem, TextInfoItem, TextItem

import json
import os
import re

'''
前端通信协议：
输入参数
-- mode:表示请求的模式，控制进入爬虫后调用的函数
        1:调用term_requests方法，对关键词进行检索
        2:翻页
        3:爬取具体的一个页面(根据PMCID)
-- term:模式1时必须传入参数，表示搜索关键词
-- currPage:模式2时必须传入参数，表示搜索页码数
-- PMCID:模式3必须传入参数，表示搜索的文章ID
'''

'''数据库状态
0:异常状态
1:已经将文章基本信息爬取至数据库
2:进入下载页面准备下载但未下载
3:文章正在下载中
4:文章已经下载至数据库但是没有其他操作
5:文章下载失败
'''


class PmcSpider(scrapy.Spider):
    # API_KEY = '8f7d71639ee4594eb9e5ed799f8712099d08'
    name = 'PMC'
    # allowed_domains = ['https://www.ncbi.nlm.nih.gov/pmc']
    # start_urls = ['https://eutils.ncbi.nlm.nih.gov/entrez/eutils/?db=PMC&id=471697397@qq.com&api_key=8f7d71639ee4594eb9e5ed799f8712099d08']
    # start_urls = ['https://www.ncbi.nlm.nih.gov/pmc/?term=city']


    cookie = "ncbi_sid=8A1A03E41D9449E1_0312SID; _ga=GA1.2.1761751111.1641628836; _ga=GA1.4.1761751111.1641628836; books.article.report=; MyNcbiSigninPreferences=O2dvb2dsZSY%3D; WebCubbyUser=%3Blogged-in%3Dfalse%3Bmy-name%3D%3Bpersistent%3Dfalse%408A1A03E41D9449E1_0312SID; pmc.article.report=classic; QSI_SI_3WoZoPQXH234L4N_intercept=true; _gid=GA1.2.771419497.1647700860; _gid=GA1.4.771419497.1647700860; WebEnv=1j9fwoslcH3mvDLQZ1mb92SjBTywtLlsjHfFx-TcMBnAr%408A1A03E41D9449E1_0312SID; ncbi_pinger=N4IgDgTgpgbg+mAFgSwCYgFwgBwEECMuAwrgMwBCATKQKwCipAbAOz4AMHnntpbt+AOgC2cfAE4QAGhABXAHYAbAPYBDVHKgAPAC6ZQlTCChzt0AF5SQpQ2CEBjSwBZDqJXYDOMoZZqHLjFzdPb2lmQ0RtIQVLbENKNksJLFQoADMVGQVdaXx8Qzyc6yx8ZkpHGjCc5yx3KBUIO0RLfACsFQVonLCsdIVa5tie9v7peMM8QhIKanomVi4Fnj4aQRFxS0o8rGNTKDMMWzsMVw8vDAA5AHlzug2DLAB3J4E5OwAjZBeFIRfkRAEAOZKGAbJIgfA0JiWXjjRjOaSkLbgyEBBH3HBlaFFEC9EZWaogUwyKDQwYgaFgmhibBieEgRwJLCMMpsAr07FsAS0AQJaSOAnyZRqDQ6Jy+LC8kCQ/IonwE/DYAzSGitECSiqGSWMJH4Ry5EAAXwNQA="
    headers = {
        'authority': 'www.ncbi.nlm.nih.gov',
        'cache-control': 'max-age=0',
        'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'upgrade-insecure-requests': '1',
        'origin': 'https://www.ncbi.nlm.nih.gov',
        'content-type': 'application/x-www-form-urlencoded',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36',
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'sec-fetch-site': 'same-origin',
        'sec-fetch-mode': 'navigate',
        'sec-fetch-user': '?1',
        'sec-fetch-dest': 'document',
        'referer': 'https://www.ncbi.nlm.nih.gov/pmc',
        'accept-language': 'zh-CN,zh;q=0.9',
        'cookie':cookie,
    }

    # 开始运行爬虫时调用该方法--入口
    def start_requests(self):
        # 判断请求模式
        mode = int(getattr(self, 'mode', 0))

        if mode == 1:
            return self.term_requests()
        if mode == 2:
            print("执行mode2")
            return self.changePage()
        if mode == 3:
            print("执行mode3")
            return self.text_requests()
        if mode == 0:
            raise ValueError("未指定请求方式！")

    # 通过关键词进行检索
    def term_requests(self):
        term = getattr(self, 'term', None)

        # 检测是否传入参数
        if not term:
            print("没有传入关键词参数")
            return

        url = 'https://www.ncbi.nlm.nih.gov/pmc/?term={}'.format(term)

        yield scrapy.Request(
            url=url,
            callback=self.term_parse,
            headers=self.headers,
            meta={'term': term}
        )

    # 执行翻页操作
    def changePage(self):
        url = 'https://www.ncbi.nlm.nih.gov/pmc/'

        # 表单原始数据 todo:把下列字典放入另外的地方
        formdata = {
            'term': 'god ',
            'EntrezSystem2.PEntrez.PMC.Pmc_PageController.PreviousPageName': 'results',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.EmailReport': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.EmailFormat': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.EmailCount': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.EmailStart': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.EmailSort': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.Email': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.EmailSubject': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.EmailText': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.EmailQueryKey': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.EmailHID': '18KR4H4UQG0DCPHtUIMRSXtiz0M_HGGHTBC_MYQgYN',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.QueryDescription': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.Key': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.Answer': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.Holding': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.HoldingFft': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.HoldingNdiSet': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.OToolValue': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.EmailTab.SubjectList': '',
            'EntrezSystem2.PEntrez.PMC.Facets.FacetsUrlFrag': 'filters=',
            'EntrezSystem2.PEntrez.PMC.Facets.FacetSubmitted': 'false',
            'EntrezSystem2.PEntrez.PMC.Facets.BMFacets': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.sPresentation': 'DocSum',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.sPageSize': '20',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.sSort': 'none',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.FFormat': 'DocSum',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.FSort': '',
            'email_format': 'DocSum',
            'email_sort': '',
            'email_count': '20',
            'email_start': '1',
            'email_address': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.FileFormat': 'docsum',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.LastPresentation': 'docsum',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.Presentation': 'docsum',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.PageSize': '20',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.LastPageSize': '20',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.Sort': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.LastSort': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.FileSort': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.Format': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.LastFormat': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_ResultsSearchController.ResultCount': '1044974',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_ResultsSearchController.RunLastQuery': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Entrez_Pager.cPage': '68',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Entrez_Pager.CurrPage': '69',  # 当前页数
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.sPresentation2': 'DocSum',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.sPageSize2': '20',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.sSort2': 'none',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.FFormat2': 'DocSum',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_DisplayBar.FSort2': '',
            'email_format2': 'DocSum',
            'email_sort2': '',
            'email_count2': '20',
            'email_start2': '1',
            'email_address2': '',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_FilterTab.CurrFilter': 'all',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_FilterTab.LastFilter': 'all',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_MultiItemSupl.Pmc_RelatedDataLinks.rdDatabase': 'rddbto',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Pmc_MultiItemSupl.Pmc_RelatedDataLinks.DbName': 'pmc',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Discovery_SearchDetails.SearchDetailsTerm': '"god"[All Fields]',
            'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.HistoryDisplay.Cmd': 'PageChanged',
            'EntrezSystem2.PEntrez.DbConnector.Db': 'pmc',
            'EntrezSystem2.PEntrez.DbConnector.LastDb': 'pmc',
            'EntrezSystem2.PEntrez.DbConnector.Term': 'city',
            'EntrezSystem2.PEntrez.DbConnector.LastTabCmd': '',
            'EntrezSystem2.PEntrez.DbConnector.LastQueryKey': '1',  # 重要参数--改变搜索关键字
            'EntrezSystem2.PEntrez.DbConnector.IdsFromResult': '',
            'EntrezSystem2.PEntrez.DbConnector.LastIdsFromResult': '',
            'EntrezSystem2.PEntrez.DbConnector.LinkName': '',
            'EntrezSystem2.PEntrez.DbConnector.LinkReadableName': '',
            'EntrezSystem2.PEntrez.DbConnector.LinkSrcDb': '',
            'EntrezSystem2.PEntrez.DbConnector.Cmd': 'PageChanged',
            'EntrezSystem2.PEntrez.DbConnector.TabCmd': '',
            'EntrezSystem2.PEntrez.DbConnector.QueryKey': '',
            'p$a': 'EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Entrez_Pager.Page',
            'p$l': 'EntrezSystem2',
            'p$st': 'pmc',
        }

        # 根据本地文件改变表单数据
        with open('./settings/formSetting.json', encoding='utf-8') as f:
            formSettings = json.loads(f.read())

        formdata.update(formSettings)

        # 判断是否传入自定义参数——页码
        currPage = getattr(self, 'currPage', None)
        if currPage:
            formdata['EntrezSystem2.PEntrez.PMC.Pmc_ResultsPanel.Entrez_Pager.CurrPage'] = currPage
        else:
            raise ValueError("未传入翻页页码!")

        # 提交表单-通过POST方法请求
        yield scrapy.FormRequest(
            url=url,
            headers=self.headers,
            formdata=formdata,
            callback=self.form_parse,
        )

    # 执行具体页面操作
    def text_requests(self):
        PMCID = getattr(self, 'PMCID', None) #type:str
        print("PMCID",PMCID)

        if not PMCID:
            raise ValueError("未传入PMCID！")

        urlList = PMCID.split(',')

        for i in urlList:
            url = 'https://www.ncbi.nlm.nih.gov/pmc/articles/{}/'.format(i)
            print("正在下载网址:",url)

            yield scrapy.Request(
                url=url,
                callback=self.text_parse,
                headers=self.headers,
                meta={'PMCID': i},
            )

    '''-------------------回调函数开始-----------------'''

    # 默认回调函数
    def parse(self, response, **kwargs):
        print("执行默认回调函数")
        print(response.text)

    # 通过关键词查找执行回调函数
    # 功能：设置cookie/表单值，重点更新关键参数  todo    :将yield TextInfoItem代码块儿放入函数中
    def term_parse(self, response):
        # 构造表单数据，为翻页做铺垫
        lastQueryKey = response.xpath(
            '''//input[@name="EntrezSystem2.PEntrez.DbConnector.LastQueryKey"]/@value''').extract_first()
        yield FormDataItem(lastQueryKey=lastQueryKey, currentPage='1', term=response.meta['term'])

        print("------------------")
        print("------------------")

        rprt_list = response.xpath('//div[@class="rprt"]')
        for rprt in rprt_list:
            # id
            PMCID = rprt.xpath('.//dl[@class="rprtid"]/dd/text()').extract_first()
            # 标题
            title = ''.join(rprt.xpath('.//div[@class="title"]/a//text()').extract())
            # 链接
            href = rprt.xpath('.//div[@class="title"]/a/@href').extract_first()
            # 作者
            author = ''.join(rprt.xpath('.//div[@class="desc"]//text()').extract())
            # 详情
            details = ''.join(rprt.xpath('.//div[@class="details"]//text()').extract())

            # 状态判断
            if title and PMCID and href:
                state = 1
            else:
                state = 0

            yield TextInfoItem(title=title, state=state, href=href,
                               author=author, details=details, PMCID=PMCID)
        print("------------------")
        print("------------------")

    # 通过POST请求进行翻页的回调函数
    def form_parse(self, response):

        print("------------------")
        print("------------------")

        rprt_list = response.xpath('//div[@class="rprt"]')
        for rprt in rprt_list:
            # id
            PMCID = rprt.xpath('.//dl[@class="rprtid"]/dd/text()').extract_first()
            # 标题
            title = ''.join(rprt.xpath('.//div[@class="title"]/a//text()').extract())
            # 链接
            href = rprt.xpath('.//div[@class="title"]/a/@href').extract_first()
            # 作者
            author = ''.join(rprt.xpath('.//div[@class="desc"]//text()').extract())
            # 细节
            details = ''.join(rprt.xpath('.//div[@class="details"]//text()').extract())

            # 状态判断
            if title and PMCID and href:
                state = 1
            else:
                state = 0

            yield TextInfoItem(title=title, state=state, href=href,
                               author=author, details=details, PMCID=PMCID)
        print("------------------")
        print("------------------")

    # 执行具体文章界面爬取回调函数
    def text_parse(self, response):
        p_list = response.xpath('//main[@id="main-content"]//div[@class="tsec sec"]')

        for p in p_list:
            PMCID = response.meta['PMCID']
            # 段落id
            secID = p.xpath('.//h2/@id').extract_first()
            # 段落名
            secTitle = p.xpath('.//h2/text()').extract_first()
            # 段落内容
            content = ''.join(p.xpath('.//p//text()').extract())

            # 状态码
            if PMCID and secID:
                state = 4
            else:
                state = 0

            yield TextItem(PMCID=PMCID, secID=secID, state=state,
                           secTitle=secTitle, content=content)
